{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ensembles\"\n",
    "\n",
    "def image_path(fig_id):\n",
    "    return os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(image_path(fig_id) + \".png\", format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.864\n",
      "0.872\n",
      "0.888\n",
      "0.904\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "log_clf = LogisticRegression()\n",
    "rnd_clf = RandomForestClassifier()\n",
    "svm_clf = SVC()\n",
    "\n",
    "vot_clf = VotingClassifier(estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)], voting='hard')\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "for clf in (log_clf, rnd_clf, svm_clf, vot_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression 0.864\n",
      "RandomForestClassifier 0.888\n",
      "SVC 0.888\n",
      "VotingClassifier 0.912\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\Administrator\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "log_clf = LogisticRegression()\n",
    "rnd_clf = RandomForestClassifier()\n",
    "svm_clf = SVC(probability=True)\n",
    "\n",
    "vot_clf = VotingClassifier(estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)], voting='soft')\n",
    "\n",
    "for clf in (log_clf, rnd_clf, svm_clf, vot_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.904"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "bag_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=100, bootstrap=True, n_jobs=-1)\n",
    "bag_clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred = bag_clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.856\n"
     ]
    }
   ],
   "source": [
    "tree_clf = DecisionTreeClassifier(random_state=42)\n",
    "tree_clf.fit(X_train, y_train)\n",
    "y_pred_tree = tree_clf.predict(X_test)\n",
    "print(accuracy_score(y_test, y_pred_tree))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "             learning_rate=1.0, loss='ls', max_depth=2, max_features=None,\n",
       "             max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "             min_impurity_split=None, min_samples_leaf=1,\n",
       "             min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "             n_estimators=3, n_iter_no_change=None, presort='auto',\n",
       "             random_state=42, subsample=1.0, tol=0.0001,\n",
       "             validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "gbrt = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=1.0, random_state=42)\n",
    "gbrt.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "             learning_rate=0.1, loss='ls', max_depth=2, max_features=None,\n",
       "             max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "             min_impurity_split=None, min_samples_leaf=1,\n",
       "             min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "             n_estimators=200, n_iter_no_change=None, presort='auto',\n",
       "             random_state=42, subsample=1.0, tol=0.0001,\n",
       "             validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbrt_slow = GradientBoostingRegressor(max_depth=2, n_estimators=200, learning_rate=0.1, random_state=42)\n",
    "gbrt_slow.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Number of features of the model must match the input. Model n_features is 2 and input n_features is 1 ",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-150456c83ed1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m121\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mplot_predictions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mgbrt\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m0.5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0.5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m0.1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0.8\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"Ensemble predictions\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"learning_rate={}, n_estimators={}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgbrt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgbrt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfontsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m14\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-12-150456c83ed1>\u001b[0m in \u001b[0;36mplot_predictions\u001b[1;34m(regressors, X, y, axes, label, style, data_style, data_label)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mplot_predictions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mregressors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstyle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"r-\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_style\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"b.\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_label\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[0mx1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m500\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mregressor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mregressor\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mregressors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_style\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdata_label\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstyle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlinewidth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-12-150456c83ed1>\u001b[0m in \u001b[0;36m<genexpr>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mplot_predictions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mregressors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstyle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"r-\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_style\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"b.\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_label\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[0mx1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m500\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mregressor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mregressor\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mregressors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_style\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdata_label\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstyle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlinewidth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\gradient_boosting.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m   2428\u001b[0m         \"\"\"\n\u001b[0;32m   2429\u001b[0m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mDTYPE\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"C\"\u001b[0m\u001b[1;33m,\u001b[0m  \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'csr'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2430\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_decision_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2431\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2432\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mstaged_predict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\gradient_boosting.py\u001b[0m in \u001b[0;36m_decision_function\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m   1584\u001b[0m         \u001b[1;31m# for use in inner loop, not raveling the output in single-class case,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1585\u001b[0m         \u001b[1;31m# not doing input validation.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1586\u001b[1;33m         \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_init_decision_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1587\u001b[0m         \u001b[0mpredict_stages\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1588\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\gradient_boosting.py\u001b[0m in \u001b[0;36m_init_decision_function\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m   1574\u001b[0m         \u001b[1;34m\"\"\"Check input and compute prediction of ``init``. \"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1575\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1576\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_validate_X_predict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1577\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_features_\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1578\u001b[0m             raise ValueError(\"X.shape[1] should be {0:d}, not {1:d}.\".format(\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\tree\\tree.py\u001b[0m in \u001b[0;36m_validate_X_predict\u001b[1;34m(self, X, check_input)\u001b[0m\n\u001b[0;32m    386\u001b[0m                              \u001b[1;34m\"match the input. Model n_features is %s and \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    387\u001b[0m                              \u001b[1;34m\"input n_features is %s \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 388\u001b[1;33m                              % (self.n_features_, n_features))\n\u001b[0m\u001b[0;32m    389\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    390\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Number of features of the model must match the input. Model n_features is 2 and input n_features is 1 "
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAEACAYAAADRH+8yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD2RJREFUeJzt3H+snmV9x/H3Zx754SklFpo6wAacZWoTS1xj1KbZMmJ0ugQY/ywgatR0QliIbmSESIYd0dnELBkjZc0EIlH4Y6MKIf5Y4hbbmCUcltTQqDWR1YHpD6jUngbLZN/98Zwuz86eq+c+Pc+5T099v5InOc91vvfN9+LhfLjuX0+qCknS//cbS92AJJ2pDEhJajAgJanBgJSkBgNSkhoMSElqMCAlqaFTQCa5NclUkhNJHpqj9lNJDiQ5muSBJOeOpVNJ6lnXFeTPgHuAB05VlOR9wB3A1cDlwJuAzy6gP0laMp0Csqoeq6qvAS/OUfoR4EtVtbeqfg78FfDRhbUoSUtjYsz7Ww98fej9HmBNkouq6v+Ea5ItwBaAycnJ33nLW94y5lYk/bp7+umnX6iq1ae7/bgDcgVwdOj9yZ8vYNbqs6p2ADsANm7cWFNTU2NuRdKvuyT7F7L9uK9iTwMrh96f/PnYmP85krToxh2Qe4ENQ+83AAdnH15L0nLQ9TafiSTnAa8BXpPkvCSjDs+/DHw8yduSvB74DPDQ2LqVpB51XUF+BniZwS08H5r5+TNJ1iaZTrIWoKq+CWwD/gXYP/P6y7F3LUk9yJnwhblepJG0GJI8XVUbT3d7HzWUpAYDUpIaDEhJajAgJanBgJSkBgNSkhoMSElqMCAlqcGAlKQGA1KSGgxISWowICWpwYCUpAYDUpIaDEhJajAgJanBgJSkBgNSkhoMSElqMCAlqcGAlKQGA1KSGgxISWowICWpwYCUpAYDUpIaDEhJajAgJanBgJSkBgNSkhoMSElqMCAlqcGAlKQGA1KSGjoFZJJVSXYmOZ5kf5IbGnXnJrk/ycEkR5I8keTS8bYsSf3ouoK8D3gFWAPcCGxPsn5E3W3Au4G3A5cALwH3jqFPSerdnAGZZBK4HrirqqarajfwOHDTiPIrgG9V1cGq+iXwKDAqSCXpjNdlBXkl8GpV7Rsa28Po4PsSsCnJJUlex2C1+Y1RO02yJclUkqnDhw/Pt29JWnRdAnIFcHTW2FHgghG1+4CfAs8DvwDeCmwdtdOq2lFVG6tq4+rVq7t3LEk96RKQ08DKWWMrgWMjarcD5wEXAZPAYzRWkJJ0pusSkPuAiSTrhsY2AHtH1G4AHqqqI1V1gsEFmncmuXjhrUpSv+YMyKo6zmAluDXJZJJNwDXAwyPKnwI+nOTCJK8FbgF+VlUvjLNpSepD19t8bgHOBw4BjwA3V9XeJJuTTA/V/TnwS+DHwGHgA8B1Y+xXknoz0aWoqo4A144Y38XgIs7J9y8yuHItScuejxpKUoMBKUkNBqQkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlJDp4BMsirJziTHk+xPcsMpat+R5LtJppMcTHLb+NqVpP5MdKy7D3gFWANcBTyZZE9V7R0uSnIx8E3gU8A/AucAl42vXUnqz5wryCSTwPXAXVU1XVW7gceBm0aUfxr4VlV9papOVNWxqvrBeFuWpH50OcS+Eni1qvYNje0B1o+ofRdwJMn3khxK8kSStaN2mmRLkqkkU4cPH55/55K0yLoE5Arg6Kyxo8AFI2ovAz4C3AasBZ4FHhm106raUVUbq2rj6tWru3csST3pcg5yGlg5a2wlcGxE7cvAzqp6CiDJZ4EXklxYVbNDVpLOaF1WkPuAiSTrhsY2AHtH1H4fqKH3J3/O6bUnSUtnzoCsquPAY8DWJJNJNgHXAA+PKH8QuC7JVUleC9wF7K6ql8bZtCT1oeuN4rcA5wOHGJxTvLmq9ibZnGT6ZFFVfQe4E3hypvbNQPOeSUk6k3W6D7KqjgDXjhjfxeAizvDYdmD7WLqTpCXko4aS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktRgQEpSQ6eATLIqyc4kx5PsT3LDHPXnJPlhkufG06Yk9W+iY919wCvAGuAq4Mkke6pqb6P+duAQsGLhLUrS0phzBZlkErgeuKuqpqtqN/A4cFOj/grgQ8Dnx9moJPWtyyH2lcCrVbVvaGwPsL5Rfy9wJ/DyqXaaZEuSqSRThw8f7tSsJPWpS0CuAI7OGjsKXDC7MMl1wERV7Zxrp1W1o6o2VtXG1atXd2pWkvrU5RzkNLBy1thK4NjwwMyh+DbgA+NpTZKWVpeA3AdMJFlXVT+eGdsAzL5Asw64HNiVBOAc4MIkB4B3VdV/jKVjSerJnAFZVceTPAZsTfIJBlexrwHeM6v0GeCNQ+/fA/wd8A7Ak4ySlp2uN4rfApzP4NadR4Cbq2pvks1JpgGq6ldVdeDkCzgC/PfM+1cXpXtJWkSd7oOsqiPAtSPGd9G417Gq/hW4bCHNSdJS8lFDSWowICWpwYCUpAYDUpIaDEhJajAgJanBgJSkBgNSkhoMSElqMCAlqcGAlKQGA1KSGgxISWowICWpwYCUpAYDUpIaDEhJajAgJanBgJSkBgNSkhoMSElqMCAlqcGAlKQGA1KSGgxISWowICWpwYCUpAYDUpIaDEhJajAgJanBgJSkBgNSkhoMSElq6BSQSVYl2ZnkeJL9SW5o1N2e5Jkkx5I8m+T28bYrSf2Z6Fh3H/AKsAa4CngyyZ6q2jurLsCHge8DvwV8O8l/VtWj42pYkvoy5woyySRwPXBXVU1X1W7gceCm2bVVta2q/r2qflVVPwK+Dmwad9OS1Icuh9hXAq9W1b6hsT3A+lNtlCTAZmD2KvPk77ckmUoydfjw4a79SlJvugTkCuDorLGjwAVzbHf3zP4fHPXLqtpRVRurauPq1as7tCFJ/epyDnIaWDlrbCVwrLVBklsZnIvcXFUnTr89SVo6XVaQ+4CJJOuGxjbQPnT+GHAHcHVVPbfwFiVpacwZkFV1HHgM2JpkMskm4Brg4dm1SW4EPge8t6p+Mu5mJalPXW8UvwU4HzgEPALcXFV7k2xOMj1Udw9wEfBUkumZ1/3jbVmS+tHpPsiqOgJcO2J8F4OLOCffXzG+1iRpafmooSQ1GJCS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlKDASlJDQakJDUYkJLUYEBKUoMBKUkNBqQkNRiQktTQKSCTrEqyM8nxJPuT3NCoS5IvJHlx5rUtScbbsiT1Y6Jj3X3AK8Aa4CrgySR7qmrvrLotwLXABqCAfwZ+Atw/nnYlqT9zriCTTALXA3dV1XRV7QYeB24aUf4R4ItV9VxVPQ98EfjoGPuVpN50WUFeCbxaVfuGxvYAvzuidv3M74br1o/aaZItDFacACeSPNOhl+XqYuCFpW5iETm/5etsnhvAby9k4y4BuQI4OmvsKHBBh9qjwIokqaoaLqyqHcAOgCRTVbWxc9fLjPNb3s7m+Z3Nc4PB/BayfZeLNNPAylljK4FjHWpXAtOzw1GSloMuAbkPmEiybmhsAzD7Ag0zYxs61EnSGW/OgKyq48BjwNYkk0k2AdcAD48o/zLw6SSXJrkE+DPgoQ597Oje8rLk/Ja3s3l+Z/PcYIHzS5ej3ySrgAeA9wIvAndU1VeTbAa+UVUrZuoCfAH4xMym/wD8hYfYkpajTgEpSb+OfNRQkhoMSElq6C0gz+bnuecxt9uTPJPkWJJnk9zed6+no+v8hurPSfLDJM/11eNCzGd+Sd6R5LtJppMcTHJbn72ejnn893lukvtn5nUkyRNJLu273/lIcmuSqSQnkjw0R+2nkhxIcjTJA0nOnWv/fa4gh5/nvhHYnmTUUzbDz3O/HfhD4E/6avI0dZ1bgA8DrwfeD9ya5I976/L0dZ3fSbcDh/pobEw6zS/JxcA3gb8HLgLeDHy7xz5PV9fP7zbg3Qz+7i4BXgLu7avJ0/Qz4B4GF5GbkrwPuAO4GrgceBPw2Tn3XlWL/gImGXxAVw6NPQz89Yja7wFbht5/HPi3Pvpc7LmN2PZvgXuXeg7jnB9wBfAD4A+A55a6/3HOD/gc8PBS97yI89sObBt6/0HgR0s9h47zvAd46BS//yrwuaH3VwMH5tpvXyvI1vPco/4v1vl57jPEfOb2v2ZOG2zmzL+Rfr7zuxe4E3h5sRsbk/nM713AkSTfS3Jo5hB0bS9dnr75zO9LwKYklyR5HYPV5jd66LEPo3JlTZKLTrVRXwE5lue5F6m3hZrP3IbdzeDf/4OL0NM4dZ5fkuuAiara2UdjYzKfz+8yBt9YdRuwFngWeGRRu1u4+cxvH/BT4HngF8Bbga2L2l1/RuUKzPF32ldAns3Pc89nbsDgxDKDc5EfrKoTi9jbOHSa38zX4m0D/rSnvsZlPp/fy8DOqnqqqn7J4BzWe5JcuMg9LsR85rcdOI/B+dVJBk/QnS0ryFG5Aqf4O4X+AvJsfp57PnMjyceYOVlcVcvhKm/X+a1jcPJ7V5IDDP64fnPmquHlPfR5uubz+X2fwRdBn3Ty5zP16AbmN78NDM7jHZn5H/e9wDtnLk4td6Ny5WBVvXjKrXo8ifoog8ORSWATgyXu+hF1n2Rwkv9SBlfS9gKfXOqTwGOa243AAeCtS93zuOfH4Kvz3jD0+iMGVxjfALxmqecwps/v94GfM/hW/dcCfwPsWur+xzi/B4F/Ai6cmd+dwPNL3f8cc5tgsOr9PIOLT+cxOM0zu+79M397b2NwF8l36HIhtceJrAK+BhxncJ7jhpnxzQwOoU/WhcGh2pGZ1zZmHok8U1/zmNuzwH8xWO6ffN2/1P2Pa36ztvk9lsFV7PnOD7iZwTm6nwNPAG9c6v7HNT8Gh9ZfYXCL1kvAbuCdS93/HHO7m8FKfvh1N4NzxNPA2qHaTwMHGZxffRA4d679+yy2JDX4qKEkNRiQktRgQEpSgwEpSQ0GpCQ1GJCS1GBASlKDASlJDf8DJJEzzQ8Apq0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 792x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_predictions(regressors, X, y, axes, label=None, style=\"r-\", data_style=\"b.\", data_label=None):\n",
    "    x1 = np.linspace(axes[0], axes[1], 500)\n",
    "    y_pred = sum(regressor.predict(x1.reshape(-1, 1)) for regressor in regressors)\n",
    "    plt.plot(X[:, 0], y, data_style, label=data_label)\n",
    "    plt.plot(x1, y_pred, style, linewidth=2, label=label)\n",
    "    if label or data_label:\n",
    "        plt.legend(loc=\"upper center\", fontsize=16)\n",
    "    plt.axis(axes)\n",
    "    \n",
    "plt.figure(figsize=(11,4))\n",
    "\n",
    "plt.subplot(121)\n",
    "plot_predictions([gbrt], X, y, axes=[-0.5, 0.5, -0.1, 0.8], label=\"Ensemble predictions\")\n",
    "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt.learning_rate, gbrt.n_estimators), fontsize=14)\n",
    "\n",
    "plt.subplot(122)\n",
    "plot_predictions([gbrt_slow], X, y, axes=[-0.5, 0.5, -0.1, 0.8])\n",
    "plt.title(\"learning_rate={}, n_estimators={}\".format(gbrt_slow.learning_rate, gbrt_slow.n_estimators), fontsize=14)\n",
    "\n",
    "save_fig(\"gbrt_learning_rate_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
